{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "81300cce",
   "metadata": {},
   "source": [
    "# Fine Tuning with PyTorch\n",
    "- Using CIFAR-10 Dataset to first train a model from scratch\n",
    "- then train a model by Fine Tuning a pretrained RESNET model\n",
    "- and then compare the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "80688c50",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ad3484f",
   "metadata": {},
   "source": [
    "#### Define Transforms and Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "48c15d77",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "57.7%IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "100.0%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./data/cifar-10-python.tar.gz to ./data\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "\n",
    "batch_size = 4\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(root='./data', train=True,\n",
    "                                        download=True, transform=transform)\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,\n",
    "                                          shuffle=True, num_workers=2)\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(root='./data', train=False,\n",
    "                                       download=True, transform=transform)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size,\n",
    "                                         shuffle=False, num_workers=2)\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat',\n",
    "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "778b3ce7",
   "metadata": {},
   "source": [
    "#### Image Samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "de7df48b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAACwCAYAAACviAzDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABTDklEQVR4nO29ebReVX3H/TvPPD/PnYfk3iSQQICEKUwBq6hpEX0dCsuqL604vHXZJlbgXVXRaldtaXjbterQhbja16JdlWKxghUVXwyIooFAJEgIZLwZb+58n3k+Z79/WJ/f/v7CvdxAeELI77PWXWvvu889Z5+99zk52d/f4BhjDCmKoiiKorQJ38nugKIoiqIopxf68aEoiqIoSlvRjw9FURRFUdqKfnwoiqIoitJW9ONDURRFUZS2oh8fiqIoiqK0Ff34UBRFURSlrejHh6IoiqIobUU/PhRFURRFaSv68aEoiqIoSlt51T4+7rjjDlq6dClFIhG6/PLLacuWLa/WpRRFURRFOYVwXo3cLt/5znfogx/8IH3961+nyy+/nL785S/TvffeSzt37qTe3t55/9bzPBodHaVkMkmO45zorimKoiiK8ipgjKFCoUCDg4Pk873E3oZ5FbjsssvM+vXrW3XXdc3g4KDZuHHjS/7toUOHDBHpj/7oj/7oj/7ozyn4c+jQoZf8tz5AJ5h6vU5bt26lW2+9tfU7n89H69ato82bNx9zfK1Wo1qt1qqb/92IufnmmykcDp/o7imKoiiK8ipQq9XoS1/6EiWTyZc89oR/fExNTZHrutTX1we/7+vroxdeeOGY4zdu3Eh/8zd/c8zvw+GwfnwoiqIoyinGQkwmTrq3y6233kq5XK71c+jQoZPdJUVRFEVRXkVO+M5Hd3c3+f1+Gh8fh9+Pj49Tf3//McfrDoeiKIqinF6c8J2PUChEa9asoU2bNrV+53kebdq0idauXXuiL6coiqIoyinGCd/5ICK65ZZb6MYbb6RLLrmELrvsMvryl79MpVKJPvzhD7/icz/wo+/jLwJ8C6XyDDT5hatPKhzktgC23fiRT7bKb3rT1dD24EM/gfolF13ZKp939lnQtnvvgVZ58zP/Bm2Lh85olTu73w1tpakdUJ+Z4XvpGxqGttUrV7XKB/IhaHtkN1Sp6flbZZ8ThDbH8rL2hETnEP4i4ONjJ/dhX8PRrlY5uRh3t3IT+6HeKBVa5diB/6a52HskCvVytYj983H/kskItGVn+BoHxg5AWzOPa8SZHWuV4zEcy0Ozk1zxDLSlYymoX/ama1rljoFBaPMHXD6N54o2Xoe+Jo656zWhns1x3/fsR3kyNzvdKod92Nczzzob6qkE7zRGInVo86z1Undr0Ob3+6Hus3Td0f2T0DY5Vm6V3/Tmi2k+nt15d6uciOC8p2KJVjkYwNdVIZeFejjK/Qs4OAbdHT2t8rqr3wVt5593OdQblgF8YRp3cGvFHJddvEbDz31v4DRTrVyGernAfS/nZ6EtPztl9aUKbdW6h8cW+bmQ77uBgYFWOZVOQ5sRz3c0wmsi6MNrpGPc1tuJ6z6eTED9Kz/cR3Nx+fv+b+6reN+EA7i2Qn6+l3BI9DXIa9Yx2FffMf+k8XmNwWtAkAk/zqX9fvntwdY1xDjbdUPYnxOFDzqLfROvJphb18h3ivXOF9dwxYma1iJ2xVr/8Tc3vkSPX5pX5ePjfe97H01OTtIXvvAFGhsbowsvvJAefPDBY4xQFUVRFEU5/XhVPj6IiDZs2EAbNmx4tU6vKIqiKMopykn3dlEURVEU5fTiVdv5eLUolVD7b7isi4eCqNlLHTGR4sAni4ZRBx+39P1HHnsU2uJR1EuXLGIttVZvQNuuXb9slT0H+7p0CWvflQJq5FMzh6EejXe2ysNLz4O2eoOnrQclVwoIW5ZajesBHwrRtkWB1IAdDxVBzxpLT4zzc7/4Qat8xsW/B219Z6+kuSjNY/Nx6aoYXt9Fe5VYLN4qj+zFsRs9yPFkBgs5aJuZxHG3ZeCY+BQf9Gx9G+0fqrNoO7IkUWmVL7wIA+w0G9xmHLSxMJaW6vOh11ehjB36xePc9+mxg9A2NLCoVZ6cPApt2595Euo9ndy/97/3GmhLJnhB1et4z9JeZXKS1/ehnaPQFonhfM1H3bJt8QfxleS3tHe3XIG2sA+f70aVxzbd2QFtay+/qlU+ewXaaYnXBDmWQUKpiM/w9AyvJ184Dm1ugJ+ZsjD6CAZxboNJXluBJtp1BCo8BrlcCdpyeTzWNgUwwian0bDfTajZl4p43lqNxzYWErY9Vv8iwh6k2cT1PB+VKl8jGsJ5Djh43pg1fJX9+HwfeZrzhHVl8FnziXrqjOWtstO7CNrK9rMn50suCvsawubDWJMgbVAC1nmkHZ2ck/liY7jW3woTD6o30Tasbt1LvYn9sd/zTdFXc4y5Ch/rvQqmLLrzoSiKoihKW9GPD0VRFEVR2sopJ7sEArg1FQzy9nxAuEvJnbMzzuYtuJ5OzK67azdvTfcP/iG0RYSL1oOWu2+hmIe2vYefa5WXnTcAbbv3Ps5989BlrVFHeWBwcHGrfGQ/uosuHehulUcbKAnV6zili1O8XxYS43Mwx/flF1uCRmwBug0+T2YAXX/ji9mL6flfPQRtjoMSTWqwhxZCdxJdE40nt0W5P5OHhHvfTLZV7E/i9fOdwl2zyFvTtSK6PJJjbZmGcQ10LVoC9VyRXV1H9qIrcr3B26LG4Bapvdss80tXGzgH5Ty7YIZ8uN09Mc5rJBzC/1PEIsIFs4fX3thBdNkdt1xUXTHmdYP1mVmWZULoIUuByNzb1scQ5GMbYgu56vJWfSaEUpw/irJHvszP4qUXXwVtl67hGENBH7pmu0L2mBxn9+vxiQnRHy4b8XelOksZlSaOVTKF+qip8bElIQ3WbPdasRNfr+O8l0r8nKRS+E6BbXyxtopCTrJdRE1SSJ41/mMpj1Sq+JzO90+K3xqvYBBludL4NNRnDx1plceew+fp2ccfa5V7u1FmCadxTVx27dtb5RWD6AJvu6+6Yr7c2txykuPHeywW2LW/mCtAW4clr3V1dUFbQLiO28+/62J/ypY0OSvctgsFlNBmcvwc1OrifWNJRkHhkp9J4/qxpUIjYzGcAHTnQ1EURVGUtqIfH4qiKIqitBX9+FAURVEUpa2ccjYf4QBqp65hzT4QQp2sowO1y0N7d7XK27Zsg7Y3XP2WVnmwG0OE/8+990B95/O/bpV9IdTNzj7/za3y9CzqxYOL+Nh0ejm01Rt4nv2Hn+Bjk3is33cO96WOOmIqguc5J8Oa+fYJ1LrJzKPjzdPk96Feu/oqDld9dGgXtNXLqJ1Wi9lWeT5nzJ07ME58Q/h6Gcs2oeyhC+bRCofEPlpEXX5o1Qrszyi7r5bKeGwwyN/miQSuu0VnLoN63c/9Gz2MGr5ruYs64nPfsTR06UJXFW7cvZ08f/29eB9+Sz+OihDlPuGSGgyxlusRtrlN7kVNrEkSGnXKsp+JZ6Tb3sI1YjsSelno2YEA99UnQtoHA7iCzl91Uat81ZVXQ1ssYtlGCT29Xsf1M3aU7Q1yedTwnSDPQSiC4xH0eP34hJ4ec3Au6x4/FwES9kwx6xpiGJsNXEDT02wrERBGbp71zITCaPsUieIamZ7id5XjCrsS6x1btcLCExGl02hzQXRs8tDfETNsixCoY1/LDRznXJPrgaFOaBvueGOr/PxTm6Ft4Ag+e1HL/qsuUis0refCreOzXymhHUXAsvPwiUSo1Rm2xTq8fy+0lTszrXIkiOEd4gl0ByeHr9F08XkCF94yjlVDhOevWHNZraG7fMQKox8T9jE+mRKgxvYi8ZiI6XAC0J0PRVEURVHain58KIqiKIrSVvTjQ1EURVGUtnLK2XykUsLXP8T6VzyE+qNf+GPny6wHZvpQ4JrNsm//v37t69iWw9DRCSt0c7eIW9EgS28roCY9kLy6VT73nLXQdujob6C+7Xm+r2ZV+NYb1mtTKNtRxhuD+i4r0vZEEw+2wwB4x4T2Rc0awjhL4wQrTHHvMNoiyIMrIm7CXByeQO3W56BGbCydXMZ3OWP1UKvsF0YWyRhq3Yt72BYgImJ5RKI8f4EAXkNEsgaf+GMiXFhGDZ5IWx2wAmQ0gqirzo5gCPXxgxzLo3dgCNqGlrDWXq3h2k4nUTO3w0PnCxinxh5nI94OAWFLYt9pUMyPj9jGwDQwpoTEZ4e5lkYxVl+nhLZ95pIzoH7lFW9olTtS+FwaK+S0DIE9MzUO9ekptgMKBNBOqljmewkJkbw/ymMQFuHDfQGMt9CwxqtE+N7K5bh/uRLaKWQyGbxmP8cSKhbRFsCOFREKoZ1CT0831A8d4rWVm8EUBHErbkxR2LIUCnhNis1t85HwZVtlrymeJ5+wsUjxeFVFSHenwWtr1VmYeiK5HWP++LPcv1Ie342+ND/70nZlpoH9mZ3h99ZkCe2Sxkb5JVuv49/5O/gdMjuF8ZpKxSmoJ5KZVjkiYthErPdzsIljbkoYI6UjyMfWRYyqfDbL/RHxZcYbIraJdc2+PoyLdSLQnQ9FURRFUdqKfnwoiqIoitJWTjnZJRDBba1qlV2JYiIrqAniVlrvALdPTOIW3JOb2X02lsCQ5asvvAjqbiPL1/DjFly1ztukTha3qY9YYcCXDK6CtmYNXeFqFd5KM4TXWJk8v1XuDGB47B2H8TxH6rz97A+itOKZucMvS47NyGidx/aVxN1l8okTJy33Z5xJpCOG8ho5eGJjfTc7Cdxa7O+ywwLj3/n9uIUbtFylA368R1vaIYPf6TL8ccm6TP2YMbAys4rw4YlUplWuVtAd8+BB3JY9eoQlkukcbuEeHGVJolLFkV11Pm5Nrz7Xrkt5zVgt+HqoiMyf9vIJhFBiNA37PPNjrAGLJIV0arn3hsMogZx/0YVQH1rEUpSIDE+OJRE1ajjOR4+irOpYmlq9gceOHmYpzNeFMlk6zeuuKcKQu55YP9aacBwcO5+1tnKWqygRUd9idM889zyeyz170D3ddr0NSjdpEYo9bdXHLXd4IqJ0mt8hMuMtOceIjHMSdvg9li2itFMVEoBnhW1vllAOyI7yc3CGD+dgUIQwN5bcNXr4eWjLVFkiypyBmY6FNy1ZHqoUCeJ7PTfDcxmPYH8iljtvfhrXWcPFtbVoMads6EqfCW1+y+V8236Ulh555BdQHx+bedEyEdHY2Dyyt5AjbQk0KdbLNe+6Yu7zLBDd+VAURVEUpa3ox4eiKIqiKG1FPz4URVEURWkrp5zNR7oDxbjAJNs4lISL4cQo6uK2F58n/AgDVqhm42JI2j0voBtsJMIaXySO3291S2wulrPQlohzKOCKi/YF0Y4RrHfvaZX7OtDmJF/jvu4cRY18wkW3SscKEe5KH1nLlcqR2j+JQ+cR7m17EEe44hlP2hSgTcpcpBOoh7rSmMSxi3OHlW428XpNYbdQKfJ1PBfPU6vz2DUIwy8bMZa2SYgnxsB2Q5XjEUlyf/wiPLZ0g21aa6ssQoLXZlgXNy6O1d7tmJJ8oIfd5jq7USNvNvhvjx5Fm5Nt27dD3fKCpeXL0cW6vxddgecjaM1fLIDPt20vc/aZGJ76jGF0tXWseXebaPfiWA+/58NnL5lGN8JghO0qdu3cCW2NCrvaBgi1/4blqjg1g2NXLOF8BS37lVQa7TjSCX6mO5Jo51IT4bI7etllNplB98ygFZvda+D69bs4zmefyWOZCOO70bFsn3r70UU33YH1PTvwubUJW+6+0ai0FRF2L5b9Xn4WXbXTIb5GXwDfv70daK9XtGy6xivoqt3h576Xi+iuWq2gnV08ytfpzOB8RSw3fGkbZ4e1L5ZFeHdhYxFP8bwfPoI2MT97iP/tuP+/H4C2I4ePQL1hveO8edIcuOLfOZ94LhqWvdPkGNpIEqnNh6IoiqIopxj68aEoiqIoSls55WQXU8Htwpi1RVgt4TZ1KimiW3azJBEODUJbTw9HOQwIN7m6Nwz1zjRvGU7NHoa20VneLgsKd61cgbfDpoq/hrbli3ELbCjxzlY5n18CbbumeHu1msBt85iI6mrfSbWG90XW9rzzEi5zxpZWjm18seJvEaFAG7IPc1CsCglEuio27YiVMvopb3XKr2tnvmyrYhvUb/2xTADsOLI/fOeuyEjpC1iZa4Uk41jSV1hsIftInMf6U6+J8lHIeg58IturK2SYX/+a115XJ26b2xFp9+xDN+6skIHSaZYE9u16AdoCVobOjnSG5qO3i/sQEDMWtNbPheegy3A8jLJHrcayR1NsKTtwHF4/HMWt+pwVeXJqCrfjyXLdlpmOPUsa9MT6qNRQ9nCtJyVpUDqNWtly+3rx+d4/gXJAocSSRFcfzmXTkg6y0+hi6Yg10dPFUkJaZDstlDiiZjiCMmYgJkIsU5bmIhK2jvX1QVs8hXJxZ5PXfqZzEbQ5/dyf0OOYRTaSwrmc6uA1ejiL7+rAXkuK24MSvSfeZOkku/5nszjvlQpLfAHxDHd28H119eA9i8S59ItHWN5/4AcPQ9vzv2H5r14Va1tEpw75eY5cGaLA8HvDiHuUkrnfapeRmU8EuvOhKIqiKEpb0Y8PRVEURVHaynF/fPz85z+nd77znTQ4OEiO49D9998P7cYY+sIXvkADAwMUjUZp3bp1tHv37hc/maIoiqIopx3HbfNRKpXoggsuoI985CN03XXXHdP+D//wD/TVr36VvvWtb9GyZcvo85//PF1zzTW0Y8cOikQiL3LG48O4eI6SyzYWrou6VFy4qdlhi/PT6NoajrIb1Hlno86azKCLX97KlHh4El3xuvpYjwuGULMf7GWNulpGPf2Jn2KmwsO7s63yc8+jXmuH5D579Rrsmx9dHNP9HKZ3YNk50Bbv4YyYRuiG0iUUrTnmtpswos0nzusFF2bzkROJUKVbmm1zERDdabis/WfzWfw74SKbsbJZRnz4Le5YNhc+YbviE8faWUz9Qju1I4/XXQwVXbLc//zC7mbxAOrXmQSfKJFAO4FEktdWIIzX94SbcqXA7Xt3YqjmnJUZNV9E99DlK5ZDvX+A18/hEfwPxvZn2K7k9974FpoP2xVZ2rKsXMl2HmetwBDYjgh532jYNkxiHVrzVcjjs3b0KLpg7h3Z3yrv3oPjk7QyvJbKqL2HLH1dROqnTBRtJVyrfxWRJXXcsJ1JqYzrZXYW+2qCvCaGl+KzX63zWFarwtVWhElPdPB6SqYxlHa1zn2oi3QFZdG/+Ww+yEoTEYqgrUhAZN01VhqCQEC8DA7xeeqzuEZDq3GNhnt4fGJlXC8zs2zn5wqjrqpwZc8WeP6cENqgpNJsA+LV8T1VtvIuvPAs/lvxo//5JdSfeYbtpopFtK9qNLk/rgjL7og4CE3HzHmssd5pTZGD4BgbkGMynZ9Yjvvj49prr6Vrr732RduMMfTlL3+Z/uqv/ore/e53ExHRv//7v1NfXx/df//99P73v/+V9VZRFEVRlFOeE2rzMTIyQmNjY7Ru3brW79LpNF1++eW0efPmF/2bWq1G+XwefhRFURRFef1yQj8+xv43ClpfH7oU9fX1tdokGzdupHQ63foZGlp4ZERFURRFUU49Tnqcj1tvvZVuueWWVj2fz8/7AeLFMERtoMm3EBTyo0yn3jNkhdo9gprevn2s3/r9+KH0tnWoR05NsAZpfKjXhoPWNR38u2xuV6s88hzacYztx76OT++3zoPTtHwph7LuCGEMkG0/+wHU94481Sr3DmGchIt//09a5Yuu/T+hzfNhrAjHsgGRSqCtFRphm1EroC4ejCwsBbfUlm3/dNkHT/RodOJoqzw9iztpdjwOIqKuIjvbLx/G2C9xK96CtPHwixTljt86r9BgjRVKPxxHPd2zJFkXb5GGh/AjvlzmA5riPpoua/o+YQsRDoh4FEG2eejsQluEQsWKfSDsfmT8klCIbap6ejFEeaUsdPp5iFoBcXoHeqDtyss5jHNHEm1gKsLmolnnerWKLwM75P6hQ5jafOcL+6E+OsF2ZAVh9zKwhPX+UDgGbTUrFX29iO+F4gzGC3EtO5dwXMRzqfDaL4iUEdLmo2rFrYmJtZWK8riGY7gmQiHx6rdiG/lEmP9YlM8r7aLCCWnDNUpzcfjQc61yphvfW+lMP9RjMcsGxcV30fS+ra1yswttR2pDOAZxH4/fhSIFgN+ywxFmWmA/RESUtuKQFItoP7N7J8ca+c1WtCXc/RzblRzYfxTasiJsPMyBT77/+EXhCrsx+b6xp8gzeB7XsvNwzfw2H3b11bD/OKE7H/39v11A4+Pj8Pvx8fFWmyQcDlMqlYIfRVEURVFev5zQj49ly5ZRf38/bdq0qfW7fD5PTzzxBK1du/ZEXkpRFEVRlFOU45ZdisUi7dnDGVdHRkZo27Zt1NnZScPDw3TTTTfR3/3d39GKFStarraDg4P0nve858T0uIJbnVTlWwiFcavTBHFbKTfDW0ehIIbzTVlhgidG8TxbtmA2z9EJ3lpM9wlpp4+3CCuzuD126DnedhsfwS1jO/Q6EVHaChN81ZXroO2yNe9olWMxDKlcLqNkNJNlqWd03xN47Hc5jHQoguO68s3vIcTaipU7cNZWniO25isFlMm8Ao/JfF++ntAgPBlq3Mf9kW6EUzM5/rsGzkHAh1vKecunt1rDee+03VeDGMqbwrgVXLZidvtFaO2aJYP4XSHfONZ2ahPd4kIifLjtDm08fHQbdR6DciUHbZUy3lfNylbZK3Ykz1jK7thbfo3ZnIsiK6jj43u2z0lEFI0u3K2+v5ullksuRtfxoX5253VFWOmgH+cgHLbcVyvYn0KB5bWRffuhbcfz6AKZszLQZqez0NbbkWmVtz+Hf1ctsLSSjuL8FHM4B47V98YsPrMVS2qJp1FqkpmO85ZE4/fjelmxjOXreBL7U6+KMO25bKsciuB5ElYIfkeE7g/7F/7/18lxlrO6ezAjcSaZgXo0wrKLaaBr9GiW5ycp/i4owrQ7TX4uOkT24ob1BgoYcc9xzFw7Ncnj/uPv/Te0PfCDn7TK42MorxkrLURYuBNL2cO1niHxCoEsFcdkDhdSN8jgJGUxO7y6kF1k1nOLVyO8+nF/fDz11FP05je/uVX/nb3GjTfeSN/85jfpU5/6FJVKJfrYxz5G2WyW3vCGN9CDDz54QmJ8KIqiKIpy6nPcHx9XX331vF9IjuPQF7/4RfriF7/4ijqmKIqiKMrrE83toiiKoihKWznprrbHSziD7oe+GXZzalZxR6bcQE2rUGU90OdDjS8YZr0t2SncnIKTUHc91iBTHfj9tqSH+7B7D+qquSN8rM+H+t+SIXTzvHQNuxguXoQhg+slTg09susxaGsIvf+db3tvq7zp0R9CWzzJmvDUjq3Qlj8Xtff0AGu0cudr+jC70E3t/BW0hZLDUI8Sa++oziKxoHQFxDkJWzYXY1XMTd207DxSMTxPNIqadanE8z41he7PhQILrZFEBtpCCXTxi1lauF/IrBUrPXizie51qRS7wfoDuCbqNRGmPWy5xYo5CAV4PcdiKHEGhFtw3HKdjIuU6OmUpbUbtI958NFHoH70MNsqzM6gbU9PJ7r3zseaCy5slc89ayW0Baz/H/nFhmtASLn+MB/baKJ9SL3G8zw+gc/zrhcwNHzRCksuTRr2H+Rnb6/lWktElInz/JS6cHVPzaBdUjTCayIcwjU5Psrr0BxB98wiPgbUJJ73gmhMWOslPoz9qZfQfidppYUPBtD9OmiFQvfFhKu8s/B/Qs5bcUmrnEqg/UU0iDZnPsuV3BPjU7fsbib24vhc1MS0AyUr5tR0BW1HAi73/fAhXBO/+MUDUP/Rj9iu43lhI+R3uH8+MRw+a9FWxdyRJ1M2hF+0TETkWG64Mpy6DEPgWvZynghZ4Hm2zYc4D83NfGrHy0V3PhRFURRFaSv68aEoiqIoSlvRjw9FURRFUdrKKWfzsfwcDJHbKLDuO7JjP7RNHUUNn3ysUddF6u5YwooXEhIpyKOoZ4ejbFPg1vDYmb2sNe/bjQExegbZdmPFmZdCW0cmA/WgJQdWCxgxdvQoxxl56Bdo8zE5i31ddR7bbsQSaOcSDrMOvfvZH0Fb3yXYv+6lZ7bK9QoaNUyPPN0q73vq29DWEPpkOMDz9eYr3khzkYyh5im1ZTtMcED4q8esdOGRIJ4nIkJix6wQy+OT6KNv22r4g6jZS52+10qr3dOBsRnscY92oCbtt0JZ5/Jor1Mqot1CTzfr5KkEnica5ftKxPH60s3dZ6fcFs+BbS+TEevFNHAu8zOst3elsT8dcbQlmY+zV5zdKkfD2Fc7rL/8n5JpCuMaKxiCz0E9vVzm2BCFPGr/hSLaDOUtcX6gF+PoRKwYGI4Prz84yDFTsmWMM3I4i89lPMxzu2JpF55nEd9poZCFtqZIYX9kgu+lgkuURkYOtsqdSXwXmTLafOSnuX81kRa+1uT+9A4vhTZfaOH/f+3MsF1bvYrj41UwjL3js+138B3be9XqVnn7JL4bdzzxa6if/YecgX3X7j3Q9t1vfY//bsc+aDt0+ADUGw0e3KBMrWCtA0/E3KhbtiuOWMF+EafGWM9l0+CzT9YYGDN/6gk7pLrn4TiTHfdD2usYsUbAzuM1Hl5dURRFURTlpdCPD0VRFEVR2sopJ7v0970A9UYHb0WXS+iSemQat+QiCd6yjDZw+31mht1ic7hDSmMHs1Dv7OXtsgShW9ov9/L2YffAFdB2+cVvapX9hFtn5SJetGiFg5aZCQ+O7ue+CffQZCdmBU2suKxVnn1hs7gmby0eHd+PbWKb2Fhbunb2UCKiYom3fguzKB2IqO3khXB7dS6KYhv26Aye17W245s13IpOpnjrXnhbU1O4WNvh4OsubtV3WBJJIoWywuw0jk/BCsmdjOOa8Fuum3KrnlyWhTqE++GqczG7czrBckrYj49uIMp9b4pQyHURarxa5rGczaLUVMhzvVzEUN5nD2PI6VTCcoMNolziNaRf4dykLNdJI8M4W9lo5Rl9Qm6zX2dNIcnMzPC9VCq4fv3HhAjnesSP2829KculOYJJMHv6WFbwinj9TAW32BPWEukbXARtDetdEIsK2bCMEtHUNL+3qiI8/+Exfv8tWYJzF2gIV2RrfKZKIjS99eyHYyJzbtfC/wlxHGu+DF6jVsP7ikV5nI3I8BpcxM/iRe99K7Qd2ncY6s89v6NVfvJXT0Hbw5sebZWrNSllzO2i6orUD+AGK9aLseQKx4dt0mXWIx6TY8KZe7bsIsOiyxDqVnh10WRsaWVhCcZfNXTnQ1EURVGUtqIfH4qiKIqitBX9+FAURVEUpa2ccjYfR8fRFqCngzXqeArd+5JpvL1kP+uaK5ejbcRzz1gan0hX7nmo1/oseXJ8H4bLDoWXtMoXrLwYO9/kP8zmMI12TfjJ1eus/2VFGu2KdezSJRi+PNGP4anPXLWuVa5OH4G2XYdYA41G0aYh5kdNdtemO1vlnuFzoW35cnaV3PcrHKuqSOfeaC4sTO/RMbSpGBH1sBU62ifEy0SQv6kTEdRZPU+4Ced4/iLi2MFBtsHo7ERdvl7dAfWgwxr6yH60NbrgAh6vc8+5CNo60hz2OpnEkOT+gPi/gcf9qws7l7KVIj0r3DOzs1NQr1ihtWt1nJ+yZb/j1lH3XjyAdgO5nGUnIGxypmZ5PM64gOYlYLktyzDOMF+izSdcHpuW23BRuM/a4dXlNYJB4QJpafOmgc9lwOV6Jo7PTCDKdi+mgvMTEOkC7Mz0gSA+MyHbTbkpnp8Gvv/iET6v28D1W65zXwtVvI8rLrwQ6qOH2H1/5OAhaOvq53spZIVNl3hvzId9bDCEtlfSNdqzUtEbg/Pjs1zHQ0Noa7RqCT6nE9O8ni9cje/G/+vPPtgqy/QATeHe61hD29+PKT7stAffvfd70Lbj+V2tciSCa6DRFL7Rtq2GDGduGW84RoZXpznrx4yd5cIs38TSTRjCr5/46Oq686EoiqIoSnvRjw9FURRFUdrKKSe7FErY5akx3rqKRnGrM9WF25m1suW25ke5ZOhM3lcqClezKGEEwn1P8zX7es+Etri1tdisZqFtNs9bwcUCXr/awGtWLLliYhq3zRuW/2gyJKLtFTE744FH/99WuT6N26nZLN/HgIhcGDEoC40f4CimEQejAS5bxtkqz1h+BrTtfO45qNOCXTBxC9nvQ/dV1zqNaKKq4W3rpF9sfxuUsGYtl7p4EmW77h6e95iI2BmN4UXrZR7LkpjLc887v1W+7NIroa1Q4O1emYGyXsUt9nyJ18SUiGQ7foRdDKemcA0EgiICbJifg2wWx2N6ks/rF/838QjHMl/mv/VEtMamT0RonAd7m1ioYuS39IlKWbhpi0zHZUvqKJfx+fJZUorfj1v8toRHRBTwWRKNh2NXqXEf8gWUIHIuX+PAYZyDmUmMKFqxsp2O+vG+hgc4qmpIpEi2I9ASEYWtUMhRIV04lpwUFxFxr/79P4D6ls1bWuW6kDHPWnkOX1/MT76I4zwf4TDLigGRObcmZCGflR42FhTz5VjSgR/nxy/cr5dY973kzCXQ9o738Bh40n1W9N12mZWRSX32ehLutP/P3/8TX0MsbseRbrmWS74jjrVcb71jZBfpwsv9kX2NWVms60L2KQuJ3O6fKx/ME4DufCiKoiiK0lb040NRFEVRlLaiHx+KoiiKorSVU87mI92JGSmzLoecTnfj7UQSnVDPTbHL40XnYFjeA4cfbJV3HXgc2vyFQaivWLGKr5nA+OGVAmu92SzqvLW67QqIWmmpJjJJepa2G0XNM2lds1pHrTIcE9lf62y7UaugC+iixayBnrP6EmgrZdHmo5RnXXrHU1ug7YUtO60a6vAhB90zg4GF+WyVhL7vDwjt2+X7lK5mdkrgXBn/LhwWdgyWRuu5aKsxNsZZW+MJtGGIR9HFz61zf2WWXZ+lyXqesJuw1oHtukpENDV1FOqTlu3PzBSGRW/U+Dxh4brp8+H6mZ7g88xms9BmLHfeitDBj0xgKP9Ymt0l4zEcj1RKuhHOjc8XtMrYZoekdoRNQ024G9csd+O6SAFQtWw1gtKGIIzjZYf+9vkxvLk/yLYKlTpef3pmf6tcmkVX3444jk/Qek5mJ9EFPhXk68ejaBsRDIi5tXwgwyJcdiLD9g79AwPQFutEOzbPCtff3dcPbekutkFpiofNNdI6Ym6qVbYnikYy0OaIiXcsN3NfQNiyGOsdJ9a244gsrpb7qLTfsXse8As3bmEXZF+nKewfjNW/P77xBmgr5nhN3vOf/wVt0yJFg51hWrq92qYkAWlzItazPZauMNUoW+H5Q0Gcu2gA13qjye0rVqC9zIlAdz4URVEURWkr+vGhKIqiKEpb0Y8PRVEURVHayiln8zF+EHWyJWey/UOHCIGdK6DG193HuuvlF18LbYt72K5j/0GMTZEto+3Gmd1sS1Iro05fqrKW2wxgHABfjHX5pofffYWi1Mi5r+kO1GBTcdbmDAlRT/iOe1Z4XePgePRY0x8MdUNbLos+37lJPs+MDEXcYPuQYBA16qaLung8lqaFUBGxMqStyKI+Pk9HBtN8Hzx8sFWOCa29qwftgMoH2A7GddFepauLjzUuzlc0hXETalZMjngENdlCPtsqj+zbA2279+xulWdmRBj0Ks6BrVLHhF7rWHETKiKWypFxjDlRsHKky9TzlSrPV7aCazsSwzGIRHgdjo/isUURXnw+atZcR8I4X3b8hVAINelSScTKqfF4RUUqetvOw/HhWgoLY4mIFR8jFMK5dCybi6CY5y4rhHpnB9o6+UTkiGjIumYT7VOCAT42IGw84gkMwR8KsB2OT6Rh70zyu7G3txfaZDqHDsuuwx/COYhY10wJWxFpuzYf25/9Zau8ZOlZeA0Rpt2p8/hI+wvXsvcKiBQEsQT23bH+f+15eB5Iy4CvRnKMsKPw85wExf/ZrVBB1N+H75dPf3ZDq3zpZZhn4Kknn4H66CiHuHeF/Znr8hpZMjwEbUuXoj2G/X7O5/G53P7ss63yzp0Yr6nRwPuams62ymefh/GbTgS686EoiqIoSls5ro+PjRs30qWXXkrJZJJ6e3vpPe95D+3cuROOqVartH79eurq6qJEIkHXX389jY+Pz3FGRVEURVFON45Ldnn00Udp/fr1dOmll1Kz2aTPfvaz9Ad/8Ae0Y8cOiv9v+Ombb76ZfvjDH9K9995L6XSaNmzYQNdddx398pe/fImzL4yM2LXf8cyBVnn1hbjVWqjhVmMmwduyz73wK2gbHmD32aRzPrRVmigzNKq8R+c6uDXu62AJIhIbhbaObt6yrORRInL96DrZEeZj603cso1ErbC3Lm61Nhq4jR63tsapG8ejaIUELxfwA3FmDO9rdprdPj2DW4J+P3/DNlwRAlu45tXrUkp4cQIiZnqxhC7Ww+fwtvYZZ2Jm3+XL2K1w+MwV0Da0ZDHUv/dddrF+7FePQVshl22VEzHcFq4LN8vxMZY24lHcKt+9hz/Qj4wehDbPkhziSXwcUync+rW972T48GnLDXdiGrfUJ6bR7dNWl0IBlGiCUe5PJy4XCstj/byl29Mltr+L6II+H1VL6gmJbKf2PXtirQuFkYzh51K6z/qsbXPpzptKoZSRiLNcGo7gOnQsN2aflETgmniRyQmUvvyWu3xaSCm2kuCJcN2OkCASlotzo47PfjLGz35SpAdoCB/Ms1ed1yrn87i2IjHuXzyJL+DGcYTd/s1OfudOlQ5AWyKJ0qlnyVTxOF4zFLBChNfxXZRJZ6AeifB5Y1GURFIJfof4PBznpnDVtsOUO2Ju/VYoeE+kMojEeM2+411vhra3XYt1yKQrJPJajZ/hRl1I9MItd2aS3+UyMXap+IZW+ZFHt0PbT3+KIRTMzpFWefcunK8V52GG4JfDcX18PPjgg1D/5je/Sb29vbR161Z64xvfSLlcjr7xjW/Q3XffTW95y1uIiOiuu+6ic845hx5//HG64oorXnGHFUVRFEU5tXlFNh+53G//h9DZ+dsvyq1bt1Kj0aB169a1jlm5ciUNDw/T5s2bX/QctVqN8vk8/CiKoiiK8vrlZX98eJ5HN910E1111VW0atVvJYuxsTEKhUKUyWTg2L6+PhobG3uRs/zWjiSdTrd+hoaGXvQ4RVEURVFeH7xsV9v169fT9u3b6bHHHnvpg+fh1ltvpVtuuaVVz+fz836AhP0YJrhR5jTxT2/ZAW2dPeg+OuuyzlqYehTa1l5ghT6fQF01YDC8eslj97auIbSV2P0096ezC3XDZf0X8jlq6BKWTqOe3Z3g+6xUULN3rFDAPgc18noNdXnbHkNOtz/AWq5rsO3Q3l1Qr1bZ5kK6vkEKZxn+2YfnbboLS7Xel0FXxbOWoV3HsqVcX3IGuu1dtIblvb7F6IYWFK6Thw+yrc2DD/1/0Pb002yrkUyiLl8s4pxkrLD26U5cd6k4z1EkIuYgznXXiBTXObzG1DTbIozPoC4/NZttlY0ICe750T4lmWR9uycpbSN4vYQTuLZCIZz3SStNvHQJ7e5YuC1AOs1z7TaFi6EVctoV4d4bjbnXuufJtOP8t8kk2j+QCM8/Mcr/UYqKsPFJy6YgIHR5Y6Uorwld3hMXsU0MXGG8ErRcih1h8xESLqHdndyfagWfrUiI11alirZYI/tRw180vLRVznT3QFvAcuN2ZbJ5aXgzD8FOXhNFB/8zWhd2ddOzvAOeqqB7b7DJ/QmLkOC1Btp1xCJc7+vBZ89njW1JvGOPdY3mZyEkrum3/w8fwL+zn6FaDe3WqhWck6B12mwOUxnU61YIB7Hua2Vx3mK2VXZE+IC6lWbgogsy0NbbdRXUN/2U+16uLPx5Xigv6+Njw4YN9MADD9DPf/5zWryYDfj6+/upXq9TNpuF3Y/x8XHq7+9/kTMRhcNhCofDL9qmKIqiKMrrj+OSXYwxtGHDBrrvvvvo4YcfpmXLlkH7mjVrKBgM0qZNm1q/27lzJx08eJDWrl17YnqsKIqiKMopzXHtfKxfv57uvvtu+v73v0/JZLJlx5FOpykajVI6naaPfvSjdMstt1BnZyelUin6xCc+QWvXrj1hni6/d+EHoe42v98q79r1ArRNjaJ7WyDOW6GTRzFj6OFnWbIZPYhbTF2L0N1u8CzeMjw8idvEOSsYas8AftsdHWUpY3wKt36XDKNLqM/wNnYsim6Lnh1ST7i6xZNyG5L3aatiKzhgbRFGw+hKmsqgzHD0KF/H74jMsFZ/XJFlN4RDd0xm0rkwhFvIsTBup3rWNmQyhduyaWvbeGYao4ZOjqFMZizX3zOH8Ty/2cnZRoslnOdQEB+ddA9vUQ724nk6krx1n8ti1NuZPG83Syknn8Mt03ErmmTTwW38uOWWm0hhW0COuTVF+QZe0w6YG6/hPCcIpRXXOrghJJGSyCA6H/b2tzxPrcZrtlDA7WUZltKOhioz+6Zi/Az1d6Ms5opon3aE095uXHedlq9/rYpj59mRU0W21+5OlBHDViTXaAzlrYi9EyzcKKXKkbB2mF0HpbimJT1NTeK78IiwwStbY3DhpVdCmz/Ifa0Vcf063sJkVCKimp21VSSf9YnouT19/AwnIygDmQbfVyKKc5nO4LEe8YUmyhjRM06ZVjkopUrhimwMj0G5hmurVuH+RCM4l2Hr/dIQ78amkBgDFX7ecyLacc1yozZCUhwf3Qv1ep3XQUM8h9kiy1mZtMi0LNyd33zt6lY5FUfTgyd37KdXynF9fNx5551ERHT11VfD7++66y760Ic+REREX/rSl8jn89H1119PtVqNrrnmGvra1772ijuqKIqiKMrrg+P6+DAiYNSLEYlE6I477qA77rjjZXdKURRFUZTXL5rbRVEURVGUtnLKZbX1RDbP0izXZ2bQpbAuwjH3dLAe5+ZQ09szxcYaHd2ohfUOoUtUKMLnff5paUdhZ88Uem2Yz9PThzYWRri62mGdGzW8r4ClvRuhCceFflzPssbnF3Gl7TDBbgO1WzujKxFRyAodLbVL2wSkIXTMpod9D4uMmXNxZBxtcvJl1NeDYZ6D1UJAfnzzz1rlwyOoh+ZmMIz97CzXz14ibAEaPF8TOZERuBNDPvf3so2MJ+wWXtjLbo3S5TFizVcohXYB5SzqvqE433Nftwi5bw2rJ3R4t4Y2H3YSVSNcOT3LJVVmLC1WRHhzj9dsoyHWaGjh/69xIasrrueG1SbdV6Mi9HnA4XpUZMft6ci0ykWRlXl3Hutpy0anI4X2VrEw33MkiHZR1QqPc0i4HjdFluaQZQwln1nbbbnZxHFtujhfdcsltCoysfocPk8+h8Eb0xkRbt1ywZTX8DV53r2GeBfSwm0+jo6zvcgi4TrfNLheOuP8/gn5hM2bFebfF8F5Loo1Mltku63J2cPQ5ljvXOeYfwpxHcasOeoUeQfCIV4HERfHNVC13+PCLqqAz5dnpabIZdFGp2KlwjBNnOfntu+GerHMc2KieGy+YduN4T36ilg/K8FhCt58YR+daHTnQ1EURVGUtqIfH4qiKIqitBX9+FAURVEUpa2ccjYf//ClL0O9VmUts7cHNWCfyCdcn2LNrTSLtz40zAHTTAj10WgU08C7M1bI7iamSHcsX/uACLXrt8JTS3/9gA814qAVpyAktGXXTr0swudWKzKss3VOoe+XrBgKfqHRZzJok9Jrhao/Ooa2CJ6xU6KLFOQiJoj0UZ+LaAzn5+BhtAGx7ytX+C9oS4U5HkQogXqxP4D1cJjPFOlA/Xj1ctZ2ZVr6aBCPLZX4mkfHMBZCzQqtnUxjPIHhIb5GRIQz75Z2N9b6CUbQFuDoAdZ9c6VZaKsIewPXmhIZvttvvxKEPZHPL14Xlk4fkbm7/QsPx9yo871AWnH6bQ6p1jUiOHbRKM6BzxrnSAiP7enhcd6/dwTa/D7Uxe1xDwawLWyt72QSn5GpKX4uGnX8u4awt7KjOsvQ9DZVYe+VFYk3CyUrjoQcciu1Qf8ApqW44MLzoB6x0ssHxMupac1PwC/mWQbymYdEL9tDVAy+UytFrPvCvA46Exi7o1Zlm4ZsGe0mZoVtS83lZ/HIKNp8zGT5mqUq2q4YYfPRacVpGRxEm4+QZQckn5FggNfoQD/+nUv4XO7fz89wTdiGpeP83ioWcH4OlrJQr9X4PdIUIdxLDX5PlRt4ns4yzmVlisdyhvCZofgr/3TQnQ9FURRFUdqKfnwoiqIoitJWTjnZpZjDrSrHx1tFw0vOhbZQHbecdoyOtsqJTrHFPcznOfe8/wPaprOY4bVcYKmlI4FhZ2eKfKxPuNp6Hl/D8WWgLRLFrdeAFb47KraQq9YWYU24Q9pb2EREZLlOiojPsIUsJaqE2PI/eyVnjp2YRHfVRpW3u40P3ckCgYCoL2yb9tJVmI12q4fbfjv3cPbggMF5fstFVn/i6BKbz+Kx2SzLKTIraL3CY+mK7e9KGaUNz9qmzXQshbaeAQ637nkoi41Z4d4rdZGVFHf1ye/nCfRP47a+O8Hb1I6LY25CKGXY7rWO2GLHOrYZkZnVtcKbC4UR07a+BA1LFmo2cf34LLlCSno+n8yubN2X2P7uHuDntKsXk1ym07jW7SzSUgay++MX2Z3tel2MnU9KK1bfGyIkQM1aB5PT+Kzl8hhivmm5O/tFttUO6xledcH50DawGN9brpXOwQj3Xju2ZEi8i5wFBJ78HSNH2H3UE/OT6cSxDFoSW7WO79G65XZanjs7ABERdXSz1NM/hO+UrkGey7qLz6Vf9C8EmYZxjVatbLWOcF8t1LiDpTGcy1gE14SJ81orVnCey2XLFdqHz4GbQcmqYMm+zRLOT8T6N2iRh+u+y4/yvinzfR6aRimZ4phC4uWgOx+KoiiKorQV/fhQFEVRFKWt6MeHoiiKoiht5ZSz+TjjrLOgPjmVbZUzAdRyn3/6CaiHYpwy+N3vvw7aRg5vb5XDaQzJfdU5vwf1w4eeaZXrhQloq/s4bbPPj9ppLMzadjqCqd1dGoI6WaF/XelD5/J5ROZwKpXxWNcKdx4TroqG7PDCqLmmhavt9GzWuiaep1Lm+6wLvVie16GFacRSaz9jKWqMwail14rx2bKH++f3UGcN+HG+fJZGGxSGC7EI972nS9jkCH09leC/LTdQE04kWU8+OoVa7kyFwz/XPbzn3KxwO7VTyIsQ2MEm141wh/QFxJhb9hhSI3dFaHhAumBaPrtND88Ukq638+BZab89kQLcTmYpbT6kzUUkynZcjrjlQJRdFfuH8FmLxND9ulDItsqzs2hUYLvIyrGqW/ZWDbEGHOFq27TnQLhCT8+wPdHMLK4Xz0hbEsuOzI/j079oUavc3YfuqsEo2ryR9dhWhSFFxAph7pMxApyF/xNyaMSyTRBh0SemcAxmJrieCOOzF7JDhvtwooeGMAx4Vz+HCAgHM9AWi7KNQ4OEjUUF32PxEP/bkYjiepkp8zNcrOF5bJvEXC4LbX7hjp5Icn+yWTzP2ASviXRXBto6BkUakR7uazKLxwYO8ljWRvEe880xqJeD3H5oFFMQLBu+kl4puvOhKIqiKEpb0Y8PRVEURVHayiknuxSK6BLV18dSS9NF98dADLfrlq7ibchIWrhnLn97qzw2+Qy0PbvzYahHnOWtcjKJ52lO8HarK1zo4jHexiqWxfYy4RZYxNrZq9XENawMtPUqbpHKXdHOzkyrHAzidNfrvK1fruIWf6WKrlXxGG83DwyiBFIsZlvlgIj8GY9jPdO5MFfbWhXHo9rMQj2Z4jEo5nGcm1Ym1DOH0H0sFsUxSER4HqIx7Fs4yJNQqeEW6cSs6J/l7lv2MCPl2H7elh0T2ZSbAT6PJ9wW3Sb+38CePscvZJcedimuivVSrwj3dGuRGEe6g/N42NEZiYhcEU3XjoDaFG0Ld8AkajSaVhn7artqB4XG6D+mzmvUHCMf8T1Ld958UfhrOvaxeF/j47w1PZ+rbdMVGa3F//M8a9zLIgrlTJafvYaMCCxkDlul8os1Ua7yu7JSw/dmwkEXdNuFOBoWUW+tqif7IyOezkMiyHKAGxbysFj744c5hEA+gJliS8RzkhIen4lulJPCee58V4d4DiyJOCTeW6EInicZZrdUR2Snzc3yfEnpIuCzpEA/ntMN4NpqeLwOuhehG2ymn2WzPuHmGsCgrjQ9wus5dxjlm8Z+7quLl6D0eTiXw4tZwsrtXXgk24WiOx+KoiiKorQV/fhQFEVRFKWt6MeHoiiKoiht5ZSz+RgfxVDnQcPClbv0KmgbvOAMqLveb1rl3Xu2QNv+fXze/gHMPrj9N5i59vABdqfNxNCmoF5j3blSRp216rDGOD6FrpqZOGqeToLrjggjXbd0aONgWzIpQix3s6uZ1MFtmxSnIEL0Ch3cziA6uAjHZ2KKs0X6Rejfnn78vnWCeJ25yJbQtWuqhC6zdqjvgIPaf18Ha6KJCOrFNeESmqvxeXIivHm5yllKaxWRlTSO7naT46ylzhbRHsS13Cprwrk1YLlO+kWGWUe4EXpNa77E/xsiaXaNdpv4d2UROtq1bDUc4e7nt11CpWutJ91HuSwzOB9H1G2qWDYP0q4jarmE+oS7qgyBbdvMHJNN2Vr79Tpq/8UyrsmqlbIglcD1Y9uAzMzgGrX77hPu6I6wlWhYdjjZWbRVqzZ4/fh9eI+ucLG27V6CYbRbCIassRS2YK7ojz22IpEvGJa44n3jSl/teWhY67BbZJAmH9pDjOZ5nMcncd1V6nwzblXYinRhuglfnJ/pWQ+z2sb9PLddUcz6K+cvZt12s4TPt2f4Xeka8Q6x1rYr1mvNw77WClyPemjX0e1nmw/ffnxGpp7E9TMywuEEygnsa2iYy51n4T0Ww9ifEPEc9XVjf6az9IrRnQ9FURRFUdqKfnwoiqIoitJW9ONDURRFUZS2csrZfJRFXIvdh/gWEotR2BxY1AH16gxrh/k86uClGbYpGNm3A9qKJeGjb2l1ERGyd3Dxxa3ydO0RPE+Z9cn+7jXQ1qhiOPOaFZ5Zhl+24y2EhJ99Io42KJ1JPq9PpIL2yNKLhX2BT0RqyJb5mpUaxgAxVtjvmognUBOisFmgRpyrZfEXQmt2rNDePiEfT1trJHsI9fyGI2Ne8H2GxdNQs7TuqojzESyizmpr+iJKOlHIChEuP/eNHeIeCQj7kJplO1IXayI7Ocrn8aEm7POjtuvz2bYReB7bJMZ1RcwNaQvgWr+QnffLg+fGjuURE6HObTsKRwaxEWvUzLO4mlYsEVcc54nOF0pW+nJhP2PH8sgKOynXih+SSGEcjVAUbUcKBQ7OUCii1h62wsjL9AQUEHNrjY+IvE7lKtsfVEUcH2kzZA/tMRFSLLsTx4/Xd7wGLZT+RfwuCobx75riPD39/FD7/dgWnOU58Yu5rBfx2MKUFcdHPJdOmN9VEYM2Zf4wjvuMdZ2owXdsJMpjUqnh+DRdngOPsG8RMbdJl+066vvxGruf4vdNcT/aGgWiIo3Hah5nb7GwcfPYjm0mC01UyOOxceL36BIPrxnpRRuZl4PufCiKoiiK0laO6+PjzjvvpPPPP59SqRSlUilau3Yt/fjHP261V6tVWr9+PXV1dVEikaDrr7+exsfH5zmjoiiKoiinG8cluyxevJhuv/12WrFiBRlj6Fvf+ha9+93vpqeffprOO+88uvnmm+mHP/wh3XvvvZROp2nDhg103XXX0S9/+csT1mG3jpuCyf6lrfJ0Afff92/aBPV4kMNeh6PoElWv8RbYbA63sSTG8Hbr0CL8fstkOCTtzBhKKTNjvM03U8NtvrBww0ok2YW4IcJBByzX23AI91p7+nA7rCPJLmyxGG6/l8q8DZjLHYI2f0KMT5G3og8dPAJtjTpvg1ar+Hf794gsnJbUQ/MkRvQHcUtShu+287HK7ea61eYK99BgUGR8tVwyG8I/1LO39cMihLsP5SXH2o6XW7bG+saX/bGrDZGdtyn0m5B1nnQIt3fTId7Wz4lw6lUZ6tty6Q0I11bj8TU8I+U+qILUJLO2SiljPlIpfk5CofCcx0nZxRGpax2HO2jEorBdsz3xdwGR7dkf5i3vch3noN7gea/VcK3bV6walGT8Ip1CpcLtxhPus5bs4hnx/0Ppjm2tda+B7619+/a3yk9s/jW0/d4bhHu6labCCDdTY2XLlSHlAzLj9jzE49x3Ge493YnvyuUrOUTATBbjhx8d4bpfrDuZTHl8H0vE8p0yMMCyfGQRjl1TZLUtNPj9N5DBrMglh++lUMVnplLmv6s3UF6LuijF+cc4/PzoZjz2hRf4367e1eiWvPSNKNGUMiytEEbup9gRljUnZnBNFmdwbc1Ms+wSCuF9Les9j14px/Xx8c53vhPqt912G9155530+OOP0+LFi+kb3/gG3X333fSWt7yFiIjuuusuOuecc+jxxx+nK6644hV3VlEURVGUU5+XbfPhui7dc889VCqVaO3atbR161ZqNBq0bt261jErV66k4eFh2rx585znqdVqlM/n4UdRFEVRlNcvx/3x8eyzz1IikaBwOEwf//jH6b777qNzzz2XxsbGKBQKUSaTgeP7+vpobGzsxU9GRBs3bqR0Ot36GRoamvNYRVEURVFOfY7b1fbss8+mbdu2US6Xo+9+97t044030qOPPvqyO3DrrbfSLbfc0qrn8/l5P0CCDobvjtQtXcpFDbaZn4B6zmXdLN4pXOiiS1tlr4mupOEY2mectZJ1s2gKbSW2PMn2EP4AanPdA5bLmvAPnZ7ZD/VeS4MNknBvs9RlR6TYDgpXvHqFx6d/cBG0TdU5xPy+6e3Q1nDRTTlfsVzNYnhf2Szr10EZllikqc+k0ZVyLip16RYs4flrilDRoPuKz+umg+e104W7nnTBtI4LCrsJEdLd81hPbsj08rZthAxPbdnzyPDhERGqPmB1ISz060KD136xjnq18I4k13LxE7cMbtwyRnqziWvN9nL0i3Dm0kV1PnyQPkDYCPnmth0xcr6sDvmEu7Ft5xKJ4LMXFe69vgCPe1DMQcPjtS5D09tUha0INaVbMPfHJ+wowF5GjKvnynu2QrELuzFjnef5HZiWorczBfWODD/vfvF82+kdpLezJxfQPExPW+HDRTqJfCkLdX+R+7B/DN/j2TKfp7sTw36X6mhrE47wmCRFqPxqhW01xkfx346qcEEPB/i9NjOzE9o8y0ffFUYnjmX7E/Xju8dUsD979nD52b2T0JaNsB3HijXn4DU68XmPeNyf0gSO89ER7oMMkV7O45ptTPOYjIkwDcvolXPcHx+hUIiWL19ORERr1qyhJ598kr7yla/Q+973PqrX65TNZmH3Y3x8nPr7++c4G1E4HKZweG4jM0VRFEVRXl+84jgfnudRrVajNWvWUDAYpE2Wh8nOnTvp4MGDtHbt2ld6GUVRFEVRXicc187HrbfeStdeey0NDw9ToVCgu+++m372s5/RT37yE0qn0/TRj36UbrnlFurs7KRUKkWf+MQnaO3aterpoiiKoihKi+P6+JiYmKAPfvCDdPToUUqn03T++efTT37yE/r93/99IiL60pe+RD6fj66//nqq1Wp0zTXX0Ne+9rUT2uFKA3WpSJHDSpdGn8KDha99wMqLHO/shrZclnWzRhNDuCdFqupEpsDXj6MjdSDOOp5XQP/rqXE+T6oXNekGoZ2JHX45k+qBNjt1eDCIfSuXClAvFjgsroliSHB3YJvVhtcfeQbH2fX4vtIp9Ml3rVDf8SguqYCIJy7jL8xFtSliXATm/juZPh3iWKBkTz6f1KitmCBNbPM5ttYt5kuEgmlacWKcgEi5bZ222ZAaPtejMdT+GzWhwTp8oroPr1GtW397zFihXmvbDHkixoTb5Ln0iXEVJlVk7PTux6RhX7gtgG3rIm0K7LaXsi+w44DIEbDbkklcv2FhA+K37Ja8poyRwjcaCOHi8izbHnPMmpx7k/mYsPHW3/qkwY6IE2OsdwHazqANSFPECrLtHeR5ggF8hm0zJWmC4/gXbttTs22z6sJ2RYTyb07w+0iGhu/pz7TKkQiOa0jYJXV3cpj7iMifELZvRoRpN8J2JBVlu6CAWBM167ms+fDvgkErVH6+D9r2b8c5eXbLSKs8O4vv477zeY26PhyrI4fw2KoV52fyAF4jN8nryUdo7uDW8N8On2UQ5wRfsUhyDMf18fGNb3xj3vZIJEJ33HEH3XHHHa+oU4qiKIqivH7R3C6KoiiKorSVUy6rrSs/l+JWhtekcGcr4h5hRwe7b139hrdDWyPPW1U7n/0ZtNVc3Po8uIc7EY7gXrTb5K2rqtjq9Gd5626qiVkCK2V0bxvs5b+tu8KV1NrjjsdQ2qlVUQYqF7g/R/diWPRAkduSKXS9K5VwK69iucIFxPZuR4a3BGWI8GIJx6BUkGHSXxxHbJwfmzSV++AKV1uyQlIbV7o4ytDnthssXiUYthabka61Mvcn31ezKRYpuFWK+7fOKyPIS29Vu3teQ/o8Wlv+L+KYbOODrXKRrdfaXpWZT70gXrNSteUb0dnjeLM0LNlOujtTndtkaG/j4LFN69hAQMy7JS9JmS6ZxLUfCLFbo3TdNJZ8EgriM9u04s+7Yt1587jMusKlOWhrWM7cUgoRuiKHhAwUsNzu7cy0REQB4RJvj4knF6K11ssV3JqfGcNQA/PhWF6N/hC6N7sN8d4q8viERMiCgJVaoOGi/CjfY1XL7TxfzOKxUUtuE3pSTmRPt9dWdxSvUbFSStTFGq3WMq3y1B6c59F96E4b6eO1duZKlAJ7l/E1Zwuj0FaUEpolPUXi6M4b7Lfc7JtibYWFtFzkeY/F8DwnAt35UBRFURSlrejHh6IoiqIobUU/PhRFURRFaSuOMWbhvlJtIJ/PUzqdps985jMa+VRRFEVRThFqtRrdfvvtlMvlKCXsbyS686EoiqIoSlvRjw9FURRFUdqKfnwoiqIoitJW9ONDURRFUZS2oh8fiqIoiqK0lddchNPfOd/UarWXOFJRFEVRlNcKv/t3eyFOtK85V9vDhw/T0NDQye6GoiiKoigvg0OHDtHixYvnPeY19/HheR6Njo6SMYaGh4fp0KFDL+kvfDqSz+dpaGhIx2cOdHzmR8dnfnR85kfHZ25O57ExxlChUKDBwUHy+ea36njNyS4+n48WL15M+XyeiIhSqdRpN4HHg47P/Oj4zI+Oz/zo+MyPjs/cnK5jk06nF3ScGpwqiqIoitJW9ONDURRFUZS28pr9+AiHw/TXf/3Xmt9lDnR85kfHZ350fOZHx2d+dHzmRsdmYbzmDE4VRVEURXl985rd+VAURVEU5fWJfnwoiqIoitJW9ONDURRFUZS2oh8fiqIoiqK0Ff34UBRFURSlrbxmPz7uuOMOWrp0KUUiEbr88stpy5YtJ7tLbWfjxo106aWXUjKZpN7eXnrPe95DO3fuhGOq1SqtX7+eurq6KJFI0PXXX0/j4+Mnqccnl9tvv50cx6Gbbrqp9bvTfXyOHDlCf/zHf0xdXV0UjUZp9erV9NRTT7XajTH0hS98gQYGBigajdK6deto9+7dJ7HH7cN1Xfr85z9Py5Yto2g0SmeeeSb97d/+LSTFOp3G5+c//zm9853vpMHBQXIch+6//35oX8hYzMzM0A033ECpVIoymQx99KMfpWKx2Ma7ePWYb3wajQZ9+tOfptWrV1M8HqfBwUH64Ac/SKOjo3CO1/P4HDfmNcg999xjQqGQ+bd/+zfz3HPPmT/90z81mUzGjI+Pn+yutZVrrrnG3HXXXWb79u1m27Zt5u1vf7sZHh42xWKxdczHP/5xMzQ0ZDZt2mSeeuopc8UVV5grr7zyJPb65LBlyxazdOlSc/7555tPfvKTrd+fzuMzMzNjlixZYj70oQ+ZJ554wuzbt8/85Cc/MXv27Gkdc/vtt5t0Om3uv/9+88wzz5h3vetdZtmyZaZSqZzEnreH2267zXR1dZkHHnjAjIyMmHvvvdckEgnzla98pXXM6TQ+P/rRj8znPvc5873vfc8QkbnvvvugfSFj8ba3vc1ccMEF5vHHHze/+MUvzPLly80HPvCBNt/Jq8N845PNZs26devMd77zHfPCCy+YzZs3m8suu8ysWbMGzvF6Hp/j5TX58XHZZZeZ9evXt+qu65rBwUGzcePGk9irk8/ExIQhIvPoo48aY3674IPBoLn33ntbxzz//POGiMzmzZtPVjfbTqFQMCtWrDAPPfSQedOb3tT6+Djdx+fTn/60ecMb3jBnu+d5pr+/3/zjP/5j63fZbNaEw2Hzn//5n+3o4knlHe94h/nIRz4Cv7vuuuvMDTfcYIw5vcdH/uO6kLHYsWOHISLz5JNPto758Y9/bBzHMUeOHGlb39vBi32cSbZs2WKIyBw4cMAYc3qNz0J4zcku9Xqdtm7dSuvWrWv9zufz0bp162jz5s0nsWcnn1wuR0REnZ2dRES0detWajQaMFYrV66k4eHh02qs1q9fT+94xztgHIh0fP7nf/6HLrnkEnrve99Lvb29dNFFF9G//uu/ttpHRkZobGwMxiedTtPll19+WozPlVdeSZs2baJdu3YREdEzzzxDjz32GF177bVEpONjs5Cx2Lx5M2UyGbrkkktax6xbt458Ph898cQTbe/zySaXy5HjOJTJZIhIx0fymstqOzU1Ra7rUl9fH/y+r6+PXnjhhZPUq5OP53l000030VVXXUWrVq0iIqKxsTEKhUKtxf07+vr6aGxs7CT0sv3cc8899Otf/5qefPLJY9pO9/HZt28f3XnnnXTLLbfQZz/7WXryySfpL/7iLygUCtGNN97YGoMXe9ZOh/H5zGc+Q/l8nlauXEl+v59c16XbbruNbrjhBiKi0358bBYyFmNjY9Tb2wvtgUCAOjs7T7vxqlar9OlPf5o+8IEPtDLb6vggr7mPD+XFWb9+PW3fvp0ee+yxk92V1wyHDh2iT37yk/TQQw9RJBI52d15zeF5Hl1yySX093//90REdNFFF9H27dvp61//Ot14440nuXcnn//6r/+ib3/723T33XfTeeedR9u2baObbrqJBgcHdXyUl02j0aA/+qM/ImMM3XnnnSe7O69ZXnOyS3d3N/n9/mM8EsbHx6m/v/8k9erksmHDBnrggQfokUceocWLF7d+39/fT/V6nbLZLBx/uozV1q1baWJigi6++GIKBAIUCATo0Ucfpa9+9asUCASor6/vtB6fgYEBOvfcc+F355xzDh08eJCIqDUGp+uz9pd/+Zf0mc98ht7//vfT6tWr6U/+5E/o5ptvpo0bNxKRjo/NQsaiv7+fJiYmoL3ZbNLMzMxpM16/+/A4cOAAPfTQQ61dDyIdH8lr7uMjFArRmjVraNOmTa3feZ5HmzZtorVr157EnrUfYwxt2LCB7rvvPnr44Ydp2bJl0L5mzRoKBoMwVjt37qSDBw+eFmP11re+lZ599lnatm1b6+eSSy6hG264oVU+ncfnqquuOsY1e9euXbRkyRIiIlq2bBn19/fD+OTzeXriiSdOi/Epl8vk8+Er0O/3k+d5RKTjY7OQsVi7di1ls1naunVr65iHH36YPM+jyy+/vO19bje/+/DYvXs3/fSnP6Wuri5oP93H5xhOtsXri3HPPfeYcDhsvvnNb5odO3aYj33sYyaTyZixsbGT3bW28md/9mcmnU6bn/3sZ+bo0aOtn3K53Drm4x//uBkeHjYPP/yweeqpp8zatWvN2rVrT2KvTy62t4sxp/f4bNmyxQQCAXPbbbeZ3bt3m29/+9smFouZ//iP/2gdc/vtt5tMJmO+//3vm9/85jfm3e9+9+vWlVRy4403mkWLFrVcbb/3ve+Z7u5u86lPfap1zOk0PoVCwTz99NPm6aefNkRk/umf/sk8/fTTLW+NhYzF2972NnPRRReZJ554wjz22GNmxYoVrxtX0vnGp16vm3e9611m8eLFZtu2bfC+rtVqrXO8nsfneHlNfnwYY8w///M/m+HhYRMKhcxll11mHn/88ZPdpbZDRC/6c9ddd7WOqVQq5s///M9NR0eHicVi5g//8A/N0aNHT16nTzLy4+N0H58f/OAHZtWqVSYcDpuVK1eaf/mXf4F2z/PM5z//edPX12fC4bB561vfanbu3HmSette8vm8+eQnP2mGh4dNJBIxZ5xxhvnc5z4H/1icTuPzyCOPvOj75sYbbzTGLGwspqenzQc+8AGTSCRMKpUyH/7wh02hUDgJd3PimW98RkZG5nxfP/LII61zvJ7H53hxjLHC+SmKoiiKorzKvOZsPhRFURRFeX2jHx+KoiiKorQV/fhQFEVRFKWt6MeHoiiKoihtRT8+FEVRFEVpK/rxoSiKoihKW9GPD0VRFEVR2op+fCiKoiiK0lb040NRFEVRlLaiHx+KoiiKorQV/fhQFEVRFKWt/P9CW4ihIRDuAwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bird  car   dog   bird \n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# functions to show an image\n",
    "\n",
    "\n",
    "def imshow(img):\n",
    "    img = img / 2 + 0.5     # unnormalize\n",
    "    npimg = img.numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# get some random training images\n",
    "dataiter = iter(trainloader)\n",
    "images, labels = next(dataiter)\n",
    "\n",
    "# show images\n",
    "imshow(torchvision.utils.make_grid(images))\n",
    "# print labels\n",
    "print(' '.join(f'{classes[labels[j]]:5s}' for j in range(batch_size)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c57692c",
   "metadata": {},
   "source": [
    "## Model Definition (from scratch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1a4081fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 6, 5)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
    "        self.fc1 = nn.Linear(16 * 5 * 5, 120)\n",
    "        self.fc2 = nn.Linear(120, 84)\n",
    "        self.fc3 = nn.Linear(84, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(F.relu(self.conv1(x)))\n",
    "        x = self.pool(F.relu(self.conv2(x)))\n",
    "        x = torch.flatten(x, 1) # flatten all dimensions except batch\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "net = Net()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f19aa3d",
   "metadata": {},
   "source": [
    "#### Optimizer and Loss Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6b6cf3c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32f244e3",
   "metadata": {},
   "source": [
    "## Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d1d516a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,  2000] loss: 2.179\n",
      "[1,  4000] loss: 1.854\n",
      "[1,  6000] loss: 1.713\n",
      "[1,  8000] loss: 1.601\n",
      "[1, 10000] loss: 1.533\n",
      "[1, 12000] loss: 1.488\n",
      "[2,  2000] loss: 1.400\n",
      "[2,  4000] loss: 1.371\n",
      "[2,  6000] loss: 1.333\n",
      "[2,  8000] loss: 1.312\n",
      "[2, 10000] loss: 1.291\n",
      "[2, 12000] loss: 1.260\n",
      "[3,  2000] loss: 1.196\n",
      "[3,  4000] loss: 1.185\n",
      "[3,  6000] loss: 1.192\n",
      "[3,  8000] loss: 1.163\n",
      "[3, 10000] loss: 1.152\n",
      "[3, 12000] loss: 1.158\n",
      "[4,  2000] loss: 1.084\n",
      "[4,  4000] loss: 1.100\n",
      "[4,  6000] loss: 1.062\n",
      "[4,  8000] loss: 1.056\n",
      "[4, 10000] loss: 1.076\n",
      "[4, 12000] loss: 1.078\n",
      "[5,  2000] loss: 0.994\n",
      "[5,  4000] loss: 1.008\n",
      "[5,  6000] loss: 1.006\n",
      "[5,  8000] loss: 0.985\n",
      "[5, 10000] loss: 1.013\n",
      "[5, 12000] loss: 1.033\n",
      "[6,  2000] loss: 0.924\n",
      "[6,  4000] loss: 0.927\n",
      "[6,  6000] loss: 0.952\n",
      "[6,  8000] loss: 0.930\n",
      "[6, 10000] loss: 0.982\n",
      "[6, 12000] loss: 0.956\n",
      "[7,  2000] loss: 0.876\n",
      "[7,  4000] loss: 0.876\n",
      "[7,  6000] loss: 0.891\n",
      "[7,  8000] loss: 0.899\n",
      "[7, 10000] loss: 0.925\n",
      "[7, 12000] loss: 0.925\n",
      "[8,  2000] loss: 0.819\n",
      "[8,  4000] loss: 0.838\n",
      "[8,  6000] loss: 0.854\n",
      "[8,  8000] loss: 0.887\n",
      "[8, 10000] loss: 0.852\n",
      "[8, 12000] loss: 0.883\n",
      "[9,  2000] loss: 0.762\n",
      "[9,  4000] loss: 0.809\n",
      "[9,  6000] loss: 0.809\n",
      "[9,  8000] loss: 0.842\n",
      "[9, 10000] loss: 0.834\n",
      "[9, 12000] loss: 0.854\n",
      "[10,  2000] loss: 0.732\n",
      "[10,  4000] loss: 0.782\n",
      "[10,  6000] loss: 0.798\n",
      "[10,  8000] loss: 0.824\n",
      "[10, 10000] loss: 0.808\n",
      "[10, 12000] loss: 0.804\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(10):  # loop over the dataset multiple times\n",
    "    running_loss = 0.0\n",
    "    for i, data in enumerate(trainloader, 0):\n",
    "        # get the inputs; data is a list of [inputs, labels]\n",
    "        inputs, labels = data\n",
    "\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        running_loss += loss.item()\n",
    "        if i % 2000 == 1999:    # print every 2000 mini-batches\n",
    "            print(f'[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}')\n",
    "            running_loss = 0.0\n",
    "\n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ab4b89e",
   "metadata": {},
   "source": [
    "## Accuracy Per Class (Training from Scratch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "dba32ee4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for class: plane is 65.3 %\n",
      "Accuracy for class: car   is 70.1 %\n",
      "Accuracy for class: bird  is 48.2 %\n",
      "Accuracy for class: cat   is 37.7 %\n",
      "Accuracy for class: deer  is 65.3 %\n",
      "Accuracy for class: dog   is 53.5 %\n",
      "Accuracy for class: frog  is 80.9 %\n",
      "Accuracy for class: horse is 57.8 %\n",
      "Accuracy for class: ship  is 80.4 %\n",
      "Accuracy for class: truck is 74.5 %\n"
     ]
    }
   ],
   "source": [
    "# prepare to count predictions for each class\n",
    "correct_pred = {classname: 0 for classname in classes}\n",
    "total_pred = {classname: 0 for classname in classes}\n",
    "\n",
    "# again no gradients needed\n",
    "with torch.no_grad():\n",
    "    for data in testloader:\n",
    "        images, labels = data\n",
    "        outputs = net(images)\n",
    "        _, predictions = torch.max(outputs, 1)\n",
    "        # collect the correct predictions for each class\n",
    "        for label, prediction in zip(labels, predictions):\n",
    "            if label == prediction:\n",
    "                correct_pred[classes[label]] += 1\n",
    "            total_pred[classes[label]] += 1\n",
    "\n",
    "\n",
    "# print accuracy for each class\n",
    "for classname, correct_count in correct_pred.items():\n",
    "    accuracy = 100 * float(correct_count) / total_pred[classname]\n",
    "    print(f'Accuracy for class: {classname:5s} is {accuracy:.1f} %')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a7e5af5",
   "metadata": {},
   "source": [
    "## Model with Pretrained RESNET18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "8483d0f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "\n",
    "class PreTrainedNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.pretrained_vgg11_bn = torchvision.models.vgg11_bn(pretrained=True)\n",
    "        self.pretrained_vgg11_bn.classifier[6] = nn.Linear(4096, 128)\n",
    "        self.fc1 = nn.Linear(128, 84)\n",
    "        self.fc2 = nn.Linear(84, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.pretrained_vgg11_bn(x))\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "pretrained_net = PreTrainedNet()\n",
    "\n",
    "import torch.optim as optim\n",
    "\n",
    "criterion_new = nn.CrossEntropyLoss()\n",
    "optimizer_new = optim.SGD(pretrained_net.parameters(), lr=0.01, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "f599310f",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainable_params = {'fc1.bias',\n",
    " 'fc1.weight',\n",
    " 'fc2.bias',\n",
    " 'fc2.weight',\n",
    " 'pretrained_vgg11_bn.classifier.6.bias',\n",
    " 'pretrained_vgg11_bn.classifier.6.weight'}\n",
    "for n, p in pretrained_net.named_parameters():\n",
    "    if n in trainable_params:\n",
    "        p.required_grad = True\n",
    "    else:\n",
    "        p.requires_grad = False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "078e8c85",
   "metadata": {},
   "source": [
    "### Training Loop (with Pretrained Model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "473c5c44",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,  2000] loss: 2.310\n",
      "[1,  4000] loss: 2.308\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(10):  # loop over the dataset multiple times\n",
    "    running_loss = 0.0\n",
    "    for i, data in enumerate(trainloader, 0):\n",
    "        # get the inputs; data is a list of [inputs, labels]\n",
    "        inputs, labels = data\n",
    "\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = pretrained_net(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        running_loss += loss.item()\n",
    "        if i % 2000 == 1999:    # print every 2000 mini-batches\n",
    "            print(f'[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}')\n",
    "            running_loss = 0.0\n",
    "\n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0772505",
   "metadata": {},
   "source": [
    "### Accuracy per Class (Training with Pretrained Model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bb43a13",
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare to count predictions for each class\n",
    "correct_pred = {classname: 0 for classname in classes}\n",
    "total_pred = {classname: 0 for classname in classes}\n",
    "\n",
    "# again no gradients needed\n",
    "with torch.no_grad():\n",
    "    for data in testloader:\n",
    "        images, labels = data\n",
    "        outputs = pretrained_net(images)\n",
    "        _, predictions = torch.max(outputs, 1)\n",
    "        # collect the correct predictions for each class\n",
    "        for label, prediction in zip(labels, predictions):\n",
    "            if label == prediction:\n",
    "                correct_pred[classes[label]] += 1\n",
    "            total_pred[classes[label]] += 1\n",
    "\n",
    "\n",
    "# print accuracy for each class\n",
    "for classname, correct_count in correct_pred.items():\n",
    "    accuracy = 100 * float(correct_count) / total_pred[classname]\n",
    "    print(f'Accuracy for class: {classname:5s} is {accuracy:.1f} %')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "torch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
